/*
* @Author: steven.wang
* @Date:   2019-03-05 17:13:24
* @Last Modified by:   steven.wang
* @Last Modified time: 2019-03-22 21:25:05
*/
const fs = require("fs");
const cheerio = require('cheerio');
const asyncQuene = require("async").queue;
const request = require('superagent');
const sleep = require('sleep');
const tools = require('./tools');
const config = require('./config');
const filePath = config.filePath;

const fileExists = {}
fileExists.countryPath = fs.existsSync(filePath.countryPath);
fileExists.provincePath = fs.existsSync(filePath.provincePath);
fileExists.cityPath = fs.existsSync(filePath.cityPath);
fileExists.countyPath = fs.existsSync(filePath.countyPath);
fileExists.townPath = fs.existsSync(filePath.townPath);
fileExists.villagePath = fs.existsSync(filePath.villagePath);

/**
 * 获取所有国家信息
 * @author steven.wang
 * @DateTime 2019-03-07T09:50:34+0800
 * @return   {[type]}                 [description]
 */
const getCountry = async () => {
	return new Promise(async (resolve, reject) => {
		let countrys = [];
		let $ = await tools.getHtmlAsync(config.countryUrl, 'utf-8');
		console.log(`download ${config.countryUrl} success`);
		$('#main_content > table table tr').each(function(idx, elem) {
			if (idx > 0) {
				countrys.push({
					shortName: $(this).find('td').eq(0).text(),
					zhName: $(this).find('td').eq(1).text(),
					enName: $(this).find('td').eq(2).text(),
					phoneCode: $(this).find('td').eq(3).text()
				})
			}
		});
		resolve(countrys);
	});
}
/**
 * 获取中国省份
 * @author steven.wang
 * @DateTime 2019-03-06T16:07:15+0800
 */
const getProvince = async () => {
	return new Promise(async (resolve, reject) => {
		let provinceUrls = [],
			$ = await tools.getHtmlAsync(config.chinaUrl);
		console.log(`download ${config.chinaUrl} success`);
		$('table.provincetable tr.provincetr>td>a').each(function(idx, elem) {
			provinceUrls.push({
				name: $(this).text(),
				url: config.chinaUrl + $(this).attr('href')
			})
		});
		let result = [], errList = [];
		let q = asyncQuene(async (aqs, cb) => {
			try {
				if(result.length%10 == 1) {
                    sleep.sleep(5);
                }
				let $ = await tools.getHtmlAsync(aqs.url);
				console.log(`download ${aqs.url} success`);
				result.push({
					code: $('.citytable .citytr td').first().text(),
					name: aqs.name,
					url: aqs.url
				});
			} catch (err) {
				errList.push(`${aqs.url}`);
				console.log(`Error : get city ${aqs.url} error : ${err}`)
			} finally {
				cb && cb();
			}
		}, config.concurrency);
		q.drain = () => {
			console.log('get province error(' + errList.length + ') list: ' + JSON.stringify(errList));
			resolve(result);
		}
		q.push(provinceUrls);
	});
};
/**
 * 获取城市
 * @author steven.wang
 * @DateTime 2019-03-06T16:08:55+0800
 * @return   {[type]}                 [description]
 */
const getData = async (data, type) => {
	return new Promise((resolve, reject) => {
		let result = [], errList = [];
		let q = asyncQuene(async (aqs, cb) => {
			try {
				if(result.length%1000 == 1) {
                    sleep.sleep(5);
                }
				let $ = await tools.getHtmlAsync(aqs.url);
				console.log(`download ${aqs.url} success`);
				$('.' + type + 'table .' + type + 'tr').each(function(idx, elem) {
					if(!$(this).find('a').eq(0).text()) {
						result.push({
							parentCode: aqs.code || '0',
							code: $(this).find('td').eq(0).text(),
							name: $(this).find('td').eq(1).text(),
							url: null
						});
					} else {
						result.push({
							parentCode: aqs.code || '0',
							code: $(this).find('a').eq(0).text(),
							name: $(this).find('a').eq(1).text(),
							url: aqs.url.replace(/\d+.html/g, $(this).find('a').eq(0).attr('href'))
						});
					}
				});
			} catch (err) {
				errList.push(aqs);
				console.log(`Error : get city ${aqs.url} error : ${err}`)
			} finally {
				cb && cb();
			}
		}, config.concurrency);
		q.drain = () => {
			console.log('get ' + type + ' error(' + errList.length + ') list: ' + JSON.stringify(errList));
			resolve(result);
		}
		let aqs = [];
		for (let item of data) {
			item.url && aqs.push(item);
		}
		q.push(aqs);
	});
}
/**
 * 主函数
 * @author steven.wang
 * @DateTime 2019-03-06T11:32:22+0800
 */
async function app() {
	let result, countrys = [],
		provinces = [],
		citys = [],
		countys = [],
		towns = [],
		villages = [];

	// 获取国家
	if (fileExists.countryPath) {
		console.log('国家已存在');
		countrys = JSON.parse(fs.readFileSync(filePath.countryPath));
	} else {
		result = await getCountry();
		for (let item of result) {
			countrys.push([item.shortName, item.zhName, item.enName, item.phoneCode]);
		}
		tools.writeJsonToFile(filePath.countryPath, result);
		// result = await tools.insertCountry(countrys);
		tools.writeJsonToFile(filePath.countryPath, result);
		console.log('存储国家成功：', result.length)
	}
	// 获取省份
	if (fileExists.provincePath) {
		console.log('省份已存在');
		provinces = JSON.parse(fs.readFileSync(filePath.provincePath));
	} else {
		result = await getProvince();
		for (let item of result) {
			provinces.push([item.name, item.code, '0']);
		}
		tools.writeJsonToFile(filePath.provincePath, result);
		// 存储省份
		// result = await tools.insertProvince(provinces);
		console.log('存储省份成功：', result.length);
		provinces = result;
	}
	// 获取城市
	if (fileExists.cityPath) {
		console.log('城市已存在');
		citys = JSON.parse(fs.readFileSync(filePath.cityPath));
	} else {
		result = await getData(provinces, 'city');
		for (let item of result) {
			citys.push([item.name, item.code, item.parentCode]);
		}
		tools.writeJsonToFile(filePath.cityPath, result);
		// 存储城市
		// result = await tools.insertCity(citys);
		console.log('存储城市成功：', result.length)
		citys = result;
	}
	// 获取区/县
	if (fileExists.countyPath) {
		console.log('区/县已存在');
		countys = JSON.parse(fs.readFileSync(filePath.countyPath));
		// let test = [];
		// for (let item of countys) {
		// 	test.push([item.name, item.code, item.parentCode]);
		// }
		// console.log(test)
		// // 存储县
		// result = await tools.insertCounty(test);
		// console.log(result)
	} else {
		result = await getData(citys, 'county');
		for (let item of result) {
			countys.push([item.name, item.code, item.parentCode]);
		}
		tools.writeJsonToFile(filePath.countyPath, result);
		console.log('存储县成功：', result.length);
		countys = result;
	}
	// 获取镇
	if (fileExists.townPath) {
		console.log('区/县已存在');
		towns = JSON.parse(fs.readFileSync(filePath.townPath));
		// let test = [];
		// for (let item of towns) {
		// 	test.push([item.name, item.code, item.parentCode]);
		// }
		// console.log(test)
		// // 存储县
		// result = await tools.insertTown(test);
		// console.log('存储区/县成功：', result.length);
		// console.log(result)
	} else {
		result = await getData(countys, 'town');
		for (let item of result) {
			towns.push([item.name, item.code, item.parentCode]);
		}
		tools.writeJsonToFile(filePath.townPath, result);
		towns = result;
	}
	// result= [{"parentCode":"222400000000","code":"222403000000","name":"敦化市","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/24/222403.html"},{"parentCode":"131100000000","code":"131171000000","name":"河北衡水高新技术产业开发区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/13/11/131171.html"},{"parentCode":"330900000000","code":"330902000000","name":"定海区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/33/09/330902.html"},{"parentCode":"350500000000","code":"350521000000","name":"惠安县","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/35/05/350521.html"},{"parentCode":"530400000000","code":"530402000000","name":"红塔区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/53/04/530402.html"},{"parentCode":"520200000000","code":"520221000000","name":"水城县","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/52/02/520221.html"},{"parentCode":"310100000000","code":"310109000000","name":"虹口区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310109.html"},{"parentCode":"340100000000","code":"340171000000","name":"合肥高新技术产业开发区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/34/01/340171.html"},{"parentCode":"330100000000","code":"330109000000","name":"萧山区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/33/01/330109.html"}];
	// result = await getData(result, 'town');
	// tools.writeJsonToFile('test.json', result);
	// console.log(result)
	// 获取社区
	if (fileExists.villagePath) {
		console.log('社区已存在');
		villages = JSON.parse(fs.readFileSync(filePath.villagePath));
		// let test = [];
		// for (let item of villages) {
		// 	test.push([item.name, item.code, item.parentCode]);
		// }
		// console.log(test)
		// // 存储社区
		// result = await tools.insertVillage(test);
		// console.log('存储社区成功：', result.length);
		// console.log(result)
	} else {
		result = await getData(towns, 'village');
		for (let item of result) {
			villages.push([item.name, item.code, item.parentCode]);
		}
		tools.writeJsonToFile(filePath.villagePath, result);
	}
	// result = [{"parentCode":"410500000000","code":"410527000000","name":"内黄县","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/41/05/410527.html"},{"parentCode":"430400000000","code":"430481000000","name":"耒阳市","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/43/04/430481.html"},{"parentCode":"220100000000","code":"220105000000","name":"二道区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220105.html"},{"parentCode":"220100000000","code":"220102000000","name":"南关区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220102.html"},{"parentCode":"220100000000","code":"220112000000","name":"双阳区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220112.html"},{"parentCode":"220100000000","code":"220106000000","name":"绿园区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220106.html"},{"parentCode":"220100000000","code":"220113000000","name":"九台区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220113.html"},{"parentCode":"220100000000","code":"220171000000","name":"长春经济技术开发区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/22/01/220171.html"},{"parentCode":"140100000000","code":"140105000000","name":"小店区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/14/01/140105.html"},{"parentCode":"310100000000","code":"310104000000","name":"徐汇区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310104.html"},{"parentCode":"310100000000","code":"310106000000","name":"静安区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310106.html"},{"parentCode":"310100000000","code":"310105000000","name":"长宁区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310105.html"},{"parentCode":"310100000000","code":"310110000000","name":"杨浦区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310110.html"},{"parentCode":"310100000000","code":"310107000000","name":"普陀区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310107.html"},{"parentCode":"310100000000","code":"310112000000","name":"闵行区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310112.html"},{"parentCode":"310100000000","code":"310113000000","name":"宝山区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310113.html"},{"parentCode":"310100000000","code":"310114000000","name":"嘉定区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310114.html"},{"parentCode":"310100000000","code":"310118000000","name":"青浦区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310118.html"},{"parentCode":"310100000000","code":"310120000000","name":"奉贤区","url":"http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/31/01/310120.html"}];
	// result = await getData(result, 'town');
	// tools.writeJsonToFile('test.json', result);
	// console.log(result)
}

app();